# # -*- coding: utf-8 -*-
import scrapy

from lj.items import LjItem
from scrapy_redis.spiders import RedisSpider
from scrapy_redis.spiders import RedisCrawlSpider # 如果是CrawlSpider方式的需要继承RedisCrawlSpider


class HouseSpider(RedisSpider):
    name = 'house'
    allowed_domains = ['lianjia.com']
    # start_urls = ['https://www.lianjia.com/city/']
    redis_key = 'lj'


    # 获取每个城市的url
    def parse(self, response):
        self.logger.info('='*30)
        city_tags = response.css(".city_list_ul a")
        for city in city_tags:
            city_url = city.css("::attr(href)").get()
            city_name = city.css("::text").get()
            item = LjItem(city=city_name)  # 城市名称
            yield scrapy.Request(city_url + 'ershoufang/',callback=self.parse_region_list,meta={'item':item})


    # 每个城市下面每个区的url
    def parse_region_list(self,response):
        # 解析区的url
        item = response.meta.get('item')
        region_tags = response.css('div[data-role="ershoufang"] div a')
        for region_tag in region_tags:
            region_url = region_tag.css('::attr(href)').get()
            region_name = region_tag.css('::text').get()
            item['region'] = region_name
            yield scrapy.Request(response.urljoin(region_url),callback=self.parse_house_page,meta={'item':item})


    # 获取翻页页码
    def parse_house_page(self,response):
        # 翻页
        import json
        item = response.meta.get('item')
        # page_data = response.css("div[comp-module='page']::attr(page-data)").get()
        # totalPage = json.loads(page_data)['totalPage']  # 总页数
        # print(totalPage)
        for x in range(1,100):
            yield scrapy.Request(response.url + 'pg' + str(x), callback=self.parse_hose_list, meta={'item': item})


    # 每个区下面每个房源的详情url
    def parse_hose_list(self,response):
        # 解析房源列表
        import re
        item = response.meta.get('item')
        detail_urls = response.css('.sellListContent li>a::attr(href)').getall()
        for detail_url in detail_urls:
            result = re.search(r"/ershoufang/\d+\.html",detail_url)
            if result:
                yield scrapy.Request(detail_url,callback=self.parse_hose,meta={'item':item})


    # 解析房源详情url
    def parse_hose(self,response):
        # 解析详情页
        item = response.meta.get('item')
        item['title'] = response.css('.main::text').get()
        item['total_price'] = response.css('.price .total::text').get()
        item['unit_price'] = response.css('.unitPriceValue::text').get()
        item['house_type'] = response.css('.content ul li:nth-child(1)::text').get()
        item['orientation'] = response.css('.content ul li:nth-child(7)::text').get()
        item['full_area'] = response.css('.content ul li:nth-child(3)::text').get()
        item['inside_area'] = response.css('.content ul li:nth-child(5)::text').get()
        item['years'] = response.css('.area .subInfo::text').get()
        yield item











